# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import codecs
import json

import pymysql
from scrapy.exporters import JsonItemExporter
from scrapy.pipelines.images import ImagesPipeline
from twisted.enterprise import adbapi


class ArticlespiderPipeline(object):
    def process_item(self, item, spider):
        return item



class JsonWithEncodingPipeline(object):
    # 自定义 json保存
    def __init__(self):
        self.file = codecs.open("article.json",'w',encoding="utf-8")

    def process_item(self,item,spider):
        lines = json.dumps(dict(item),ensure_ascii=False) + "\n"
        self.file.write(lines)
        return item
    def spider_closed(self,spider):
        self.file.close()

class JsonExportPipeline(object):
    #调用 scrapy提供的json export导出json 文件
    def __init__(self):
        self.file = open("article_2.json","wb")
        self.exporter = JsonItemExporter(self.file,encoding="utf-8",ensure_ascii=False)
        self.exporter.start_exporting()
    def close_spider(self,spider):
        self.exporter.finish_exporting()
        self.file.close()
    def process_item(self,item,spider):
        self.exporter.export_item(item)
        return item

#下载 封面图片 并 保存 下载路径
class ArticImagePipline(ImagesPipeline):
    def item_completed(self, results, item, info):
        if "front_img" in item:
            for ok,value in results:
                image_file_path = value["path"]
            item["front_img_path"] = image_file_path
        return item
#同步方式插入数据
class MysqlPipeline(object):
    def __init__(self):
        self.conn = pymysql.connect(host="39.96.15.6", user="root",password="123456",database="spider",charset="utf8")
        self.cursor = self.conn.cursor()

    def process_item(self,item,spider):
        insqrt_sql = """
        insert into article_spider(title,url,create_date,fav_nums)
        values(%s,%s,%s,%s)
        """
        self.cursor.execute(insqrt_sql,(item["title"],item["url"],item["create_date"],item["fav_nums"]))
        self.conn.commit()


# 异步方式 插入数据
class MysqlTwistedPipeline(object):

    def __init__(self,dbpool):
        self.dbpool = dbpool

    @classmethod
    def from_settings(cls,settings):
        dbparms = dict(
            host = settings["MYSQL_HOST"],
            db = settings["MYSQL_DBNAME"],
            user = settings["MYSQL_USER"],
            passwd = settings["MYSQL_PASSWORD"],
            charset = "utf8",
            cursorclass = pymysql.cursors.DictCursor,
            use_unicode = True,
        )
        dbpool = adbapi.ConnectionPool("pymysql",**dbparms)
        return cls(dbpool)

    def process_item(self,item,spider):
        query = self.dbpool.runInteraction(self.do_insert,item)
        query.addErrback(self.handler_error)


    def handler_error(self,failure):
        print(failure)

    def do_insert(self,cursor,item):
        insqrt_sql = """
                insert into article_spider(title,create_date,url,url_object_id,front_img,front_img_path,praise_nums,cpmment_nums,fav_nums,tags,content)
                values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
                """
        cursor.execute(insqrt_sql, (item["title"], item["create_date"],item["url"],item["url_object_id"],item["front_img"],item["front_img_path"],item["praise_nums"],item["cmment_nums"],item["fav_nums"],item["tags"], item["content"]))